merge_and_slice_data <- function(directory,output_path, num_users = 915) {
  csv_files <- list.files(directory, pattern = "*.csv", full.names = TRUE)
    read_csv_with_filename <- function(file_path) {
    df <- read.csv(file_path)
    filename <- file_path_sans_ext(basename(file_path))
    df$source_file <- filename
    return(df)
  }
    combined_data <- csv_files %>% lapply(read_csv_with_filename) %>% bind_rows()
  
    user_names <- unique(combined_data$source_file)
    select_users <- sample(user_names, num_users)
    sliced_data <- combined_data %>% filter(source_file %in% select_users) %>% group_by(source_file) %>%
                   slice_head(n = 3) %>% ungroup()
  
    write.csv(sliced_data, output_path, row.names = FALSE)
}

import demoji
import re
import ast
from typing import Union
import pandas as pd
from bs4 import BeautifulSoup

emoticon_data_path="./emoticon_dict.json"

class TwitterDataPrep:
    def __init__(self, emoticon_data_path=None):
        self.emoticons_dict = {}
        if emoticon_data_path:
            self.load_emoticons(emoticon_data_path)

    def load_emoticons(self, emoticon_data_path):
        """
        Load the emoticons dictionary from a JSON file.
        :param emoticon_data_path: Path to the emoticons JSON file.
        """
        import json
        with open(emoticon_data_path, 'r') as file:
            self.emoticons_dict = json.load(file)

    def clean_text(self, text: str) -> str:
       
        pat1 = r'@[^ ]+'  # remove username @
        pat2 = r'https?://[A-Za-z0-9./]+'  # remove urls
        pat3 = r'\'s'  # remove apostrophe
        pat4 = r'\#\w+'  # remove hashtag
        pat5 = r'&amp '  # remove unicode `&`
        pat7 = r'RT'  # remove RT / retweet
        pat8 = r'www\S+'  # remove link www
        combined_pat = r'|'.join((pat1, pat2, pat3, pat4, pat5, pat7, pat8))  # combine all patterns
        text = re.sub(combined_pat, "", text)
        text = re.sub(r'\s+', ' ', text)  # remove extra spaces
        return text.strip()

    def _parse_bytes(self, field: Union[str, ast.AST]) -> Union[str, ast.AST]:
      
        result = field
        try:
            result = ast.literal_eval(field)
        finally:
            return result.decode() if isinstance(result, bytes) else field

    def replace_emoticons(self, text: str) -> str:
      
        for emoticon, context in self.emoticons_dict.items():
            text = text.replace(emoticon, ' ' + context + ' ')
            text = re.sub(' +', ' ', text)
        return text

    def replace_emojis(self, text: str) -> str:
       
        for emoji, context in demoji.findall(text).items():
            text = text.replace(emoji, ' ' + context + ' ')
            text = re.sub(' +', ' ', text)
        return text

    def preprocess_data(self, df: pd.DataFrame) -> pd.DataFrame:
        
        df['tweet'] = df['tweet'].apply(self._parse_bytes)
        df['tweet'] = df['tweet'].str.normalize('NFKD')
        df['tweet'] = df['tweet'].map(self.clean_text)
        df['tweet'] = df['tweet'].apply(lambda x: self.replace_emoticons(x))
        df['tweet'] = df['tweet'].apply(lambda x: self.replace_emojis(x))
        return df

def preprocess_twitter_data(input_path: str = None, output_path: str = None, df: pd.DataFrame = None, emoticon_data_path: str = None):
  
    twitter_data_prep = TwitterDataPrep(emoticon_data_path=emoticon_data_path)
    
    if input_path:
        df = pd.read_csv(input_path)
    
    if df is not None:
        preprocessed_data = twitter_data_prep.preprocess_data(df)
        if output_path:
            preprocessed_data.to_csv(output_path, index=False)
        return preprocessed_data
    else:
        raise ValueError("'input_path' is not provided.")

create_embeddings <- function(input_path, output_path, api_key = Sys.getenv("OPENAI_API_KEY"), batch_size = 100) {
  df <- read.csv(input_path)
  get_embeddings <- function(texts) {
    tryCatch({
      response <- POST(
        url = "https://api.openai.com/v1/embeddings",
        add_headers(
          "Authorization" = paste("Bearer", api_key),
          "Content-Type" = "application/json"
        ),
        body = toJSON(list(
          model = "text-embedding-ada-002",
          input = texts
        ), auto_unbox = TRUE),
        encode = "json"
      )
      stop_for_status(response)
      content <- content(response, "parsed")
      return(lapply(content$data, function(item) item$embedding))
    }, error = function(e) {
      message("Error getting embeddings: ", e$message)
      return(NULL)
    })
  }
  
  # Create a progress bar
  num_batches <- ceiling(nrow(df) / batch_size)
  pb <- progress_bar$new(total = num_batches, format = "[:bar] :percent :eta")
  
  # Process tweets in batches using lapply
  batch_indices <- split(seq_len(nrow(df)), ceiling(seq_len(nrow(df))/batch_size))
  embeddings <- lapply(batch_indices, function(indices) {
    batch_embeddings <- get_embeddings(df$tweets[indices])
    pb$tick()
    return(batch_embeddings)
  })
  
  # Flatten the list of embeddings
  embeddings <- unlist(embeddings, recursive = FALSE)
  
  # Remove NULL values and convert embeddings list to dataframe
  embeddings <- Filter(Negate(is.null), embeddings)
  embeddings_df <- do.call(rbind, embeddings) %>% as.data.frame()
  colnames(embeddings_df) <- paste0("E", 1:ncol(embeddings_df))
  result_df <- cbind(df, embeddings_df)
  result_df <- result_df %>% mutate(across(everything(), as.character))
  write.csv(result_df, output_path, row.names = FALSE)
  message("Embeddings created and saved successfully.")
}

plot_columns <- function(dataset, no_of_cols, hvt.results) {
  cols_to_plot <- colnames(dataset)[no_of_cols]
  plots <- lapply(cols_to_plot, function(col_name) {
    print(plotHVT(hvt.results, hmap.cols = col_name, child.level = 1, 
                  plot.type = '2Dheatmap', centroid.size = 0.2))
  })
  invisible(plots)}

create_and_write_clusters <- function(cluster_numbers, cluster_method) {
  cluster_list <- lapply(cluster_numbers, function(i) {
    subset(cluster_method$names.column, cluster_method$clusters == i)
  })
  invisible(list2env(setNames(cluster_list, paste0("clust_", cluster_numbers)), envir = .GlobalEnv))
  write_cluster_to_file <- function(cluster, cluster_name) {
    processed_text <- paste0("'", 
                             gsub("[[:punct:]]", "", paste(cluster, collapse = " ")),
                             "'")
    filename <- file.path("./twitter_data/",paste0(cluster_name, ".txt"))
    writeLines(processed_text, filename)
  }
  cluster_names <- paste0("clust_", cluster_numbers)
  invisible(lapply(cluster_names, function(name) {
    cluster_data <- get(name)
    write_cluster_to_file(cluster_data, name)
  }))}

get_mode <- function(v) {
  uniqv <- unique(v)
  uniqv[which.max(tabulate(match(v, uniqv)))]
}

get_cluster_domains <- function(k, iterations = 10) {
  url <- "http://172.25.1.85:31969/v1/chat/completions"
  read_file_content <- function(filename) {
 filepath <- file.path("./twitter_data/", filename)
     readLines(filepath, warn = FALSE)
 }
  
  cluster_names <- paste0("clust_", 1:k)
  content_list <- lapply(paste0(cluster_names, ".txt"), read_file_content)
  
  prompt <- "Suggest a single word/domain that best describes the topics in the following text. Your response should be just one word. This is for cluster {cluster_number}. Ensure that your suggestion is unique and not used for any other cluster. Previously used domains are: {used_domains}. Do not repeat any of these."
  
  send_request <- function(content, cluster_number, used_domains) {
    full_prompt <- gsub("\\{cluster_number\\}", cluster_number, prompt)
    full_prompt <- gsub("\\{used_domains\\}", paste(used_domains, collapse = ", "), full_prompt)
    full_content <- paste(full_prompt, content)
    
    request_body <- paste0('{
      "model": "gpt-3.5-turbo",
      "temperature":0,
      "messages": [
        {
          "role": "user",
          "content": "', full_content, '"
        }
      ]
    }')
    response <- POST(url,
                     body = request_body,
                     add_headers(.headers = c(
                       `Content-Type` = "application/json",
                       `Authorization` = "Bearer sk-musigma"
                     )),
                     content_type("application/json"))
    if (response$status_code == 200) {
      response_content <- content(response, as = "parsed")
      domain <- response_content$choices[[1]]$message$content
      domain
    } else {
      stop(paste("Error:", response$status_code, response$content))
    }
  }
  
  results <- matrix(nrow = iterations, ncol = k)
  colnames(results) <- paste0("Domain_name_for_cluster_", 1:k)
  
  results <- t(replicate(iterations, {
    used_domains <- character(0)
    unlist(lapply(seq_len(k), function(j) {
      domain <- send_request(content_list[[j]], j, used_domains)
      used_domains <<- c(used_domains, domain)
      domain
    }))
  }))
  
  results <- as.data.frame(results)
  colnames(results) <- paste0("Domain_name_for_cluster_", 1:k)  # Explicitly set column names

  mode_results <- sapply(results, get_mode)
  results <- rbind(results, Mode = mode_results)
  formatted_table <- displayTable(results, scroll = TRUE)
  return(list(raw_data = results, formatted_table = formatted_table, mode = mode_results))
}